import requests
import time
import threading
from lxml import etree
import pandas as pd
import re
from concurrent.futures import ThreadPoolExecutor, as_completed
cookies = {
    'sajssdk_2015_cross_new_user': '1',
    'sensorsdata2015jssdkchannel': '%7B%22prop%22%3A%7B%22_sa_channel_landing_url%22%3A%22%22%7D%7D',
    'wz_uuid': 'X/fe36031e2352d91d31adfe63416a305b',
    'Hm_lvt_9ea3e7293b7c088e0d2c88874b63e7dd': '1747617767',
    'HMACCOUNT': 'A735DDA42AC02E6E',
    'SSO_SESSION_ID': 'ZWVjOGY0MmYtZDExMi00YWZmLTg4ZWQtZDdlNDVkYjMwMWY0',
    'ticket': 'eyJhbGciOiJIUzUxMiJ9.ZXlKNmFYQWlPaUpFUlVZaUxDSmhiR2NpT2lKa2FYSWlMQ0psYm1NaU9pSkJNVEk0UTBKRExVaFRNalUySW4wLi5odVhicExqVDBmTG1STXQ2b1ZPYnZnLkp2R3BYa2FlcHVaN1ZjUHkySjBheVRNVlZyWURSdmhmazREdWhrc2s2RTQwQWZvVDNhZXA0REkyRVF1ek1zekJOZEM2STFqZzJOSTJjRV9GX3VRZlZDNVZDWEFQSmtrS2tMSTZpSW5CM0FjYU82TFNjV2hrUTVva1htN29UTU9SbjZCVWpmcDdyRnl3RG5XRHo2NDRiSm1vT28xMGx2VnFGczM2OUlmeUltVmlfZjFmMU5GaHVYTk1mSzJrd2lVX2xGaUR5YThmd3dLZFpCaG1Wa3NMN3ktWnVMNDIzOWk3aUJYSHpTTkw5NjJSZVI1aTBPNjJyeEk1SUZmeWtrS0IuRGd4aE1TSE5YcHNRdlRiX3VZVXpxUQ.K0Ytme5n24LKre-EXu0nz7CClT1iT2HX_bt_vmFnppc3xwzHCGia43d3WyE3JZVksuo_N5EjcDxj3s3HgI-W8g',
    '_g_t_': '96f67a30d3211b400f55ae4ed110f261',
    'token': 'eyJhbGciOiJIUzUxMiJ9.ZXlKNmFYQWlPaUpFUlVZaUxDSmhiR2NpT2lKa2FYSWlMQ0psYm1NaU9pSkJNVEk0UTBKRExVaFRNalUySW4wLi5odVhicExqVDBmTG1STXQ2b1ZPYnZnLkp2R3BYa2FlcHVaN1ZjUHkySjBheVRNVlZyWURSdmhmazREdWhrc2s2RTQwQWZvVDNhZXA0REkyRVF1ek1zekJOZEM2STFqZzJOSTJjRV9GX3VRZlZDNVZDWEFQSmtrS2tMSTZpSW5CM0FjYU82TFNjV2hrUTVva1htN29UTU9SbjZCVWpmcDdyRnl3RG5XRHo2NDRiSm1vT28xMGx2VnFGczM2OUlmeUltVmlfZjFmMU5GaHVYTk1mSzJrd2lVX2xGaUR5YThmd3dLZFpCaG1Wa3NMN3ktWnVMNDIzOWk3aUJYSHpTTkw5NjJSZVI1aTBPNjJyeEk1SUZmeWtrS0IuRGd4aE1TSE5YcHNRdlRiX3VZVXpxUQ.K0Ytme5n24LKre-EXu0nz7CClT1iT2HX_bt_vmFnppc3xwzHCGia43d3WyE3JZVksuo_N5EjcDxj3s3HgI-W8g',
    'accessToken': 'eyJhbGciOiJIUzUxMiJ9.ZXlKNmFYQWlPaUpFUlVZaUxDSmhiR2NpT2lKa2FYSWlMQ0psYm1NaU9pSkJNVEk0UTBKRExVaFRNalUySW4wLi5odVhicExqVDBmTG1STXQ2b1ZPYnZnLkp2R3BYa2FlcHVaN1ZjUHkySjBheVRNVlZyWURSdmhmazREdWhrc2s2RTQwQWZvVDNhZXA0REkyRVF1ek1zekJOZEM2STFqZzJOSTJjRV9GX3VRZlZDNVZDWEFQSmtrS2tMSTZpSW5CM0FjYU82TFNjV2hrUTVva1htN29UTU9SbjZCVWpmcDdyRnl3RG5XRHo2NDRiSm1vT28xMGx2VnFGczM2OUlmeUltVmlfZjFmMU5GaHVYTk1mSzJrd2lVX2xGaUR5YThmd3dLZFpCaG1Wa3NMN3ktWnVMNDIzOWk3aUJYSHpTTkw5NjJSZVI1aTBPNjJyeEk1SUZmeWtrS0IuRGd4aE1TSE5YcHNRdlRiX3VZVXpxUQ.K0Ytme5n24LKre-EXu0nz7CClT1iT2HX_bt_vmFnppc3xwzHCGia43d3WyE3JZVksuo_N5EjcDxj3s3HgI-W8g',
    'creditNo': '%22%22',
    'param_sign': 'roYuB3',
    'Hm_lpvt_9ea3e7293b7c088e0d2c88874b63e7dd': '1747619300',
    'smidV2': '202505191016307c10a18d5e8a15a50dda3fc1039d489f00ea7a21ddcd9e570',
    '_bl_uid': 'smmtqayquXdgR4fq7006g2h9IX2I',
    'Hm_lvt_8b5ea374777742fd73d0561bd1dfca6d': '1747620994',
    'acw_tc': 'f5d36357174762107277210433481bd19eb244908197885b08a7a18a2916d7',
    'sensorsdata2015jssdkcross': '%7B%22distinct_id%22%3A%22f7b1dcc0ddbf455d833fa99414f2780e%22%2C%22first_id%22%3A%22196e622d7c2e7c-0287f8b21c83c1e-26011f51-1327104-196e622d7c31b05%22%2C%22props%22%3A%7B%22%24latest_traffic_source_type%22%3A%22%E7%9B%B4%E6%8E%A5%E6%B5%81%E9%87%8F%22%2C%22%24latest_search_keyword%22%3A%22%E6%9C%AA%E5%8F%96%E5%88%B0%E5%80%BC_%E7%9B%B4%E6%8E%A5%E6%89%93%E5%BC%80%22%2C%22%24latest_referrer%22%3A%22%22%7D%2C%22identities%22%3A%22eyIkaWRlbnRpdHlfY29va2llX2lkIjoiMTk2ZTYyMmQ3YzJlN2MtMDI4N2Y4YjIxYzgzYzFlLTI2MDExZjUxLTEzMjcxMDQtMTk2ZTYyMmQ3YzMxYjA1IiwiJGlkZW50aXR5X2xvZ2luX2lkIjoiZjdiMWRjYzBkZGJmNDU1ZDgzM2ZhOTk0MTRmMjc4MGUifQ%3D%3D%22%2C%22history_login_id%22%3A%7B%22name%22%3A%22%24identity_login_id%22%2C%22value%22%3A%22f7b1dcc0ddbf455d833fa99414f2780e%22%7D%2C%22%24device_id%22%3A%22196e622d7c2e7c-0287f8b21c83c1e-26011f51-1327104-196e622d7c31b05%22%7D',
    'x-web-ip': '171.218.206.136, 47.121.110.187, 47.106.31.167, 100.121.99.209',
    '.thumbcache_de0d870e3139ba2368b2e7ea8f11063c': 'hZVjd47Cv1sJHhGMmTvVn0f9DpivVn6IF/IytbVBOsjFSd0QEKKRar19TbnFCtp96Hi7/tlo0p6ovKPk4j5lOw%3D%3D',
    'Hm_lpvt_8b5ea374777742fd73d0561bd1dfca6d': '1747621872',
}

headers = {
    'accept': 'text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,image/webp,image/apng,*/*;q=0.8,application/signed-exchange;v=b3;q=0.7',
    'accept-language': 'zh-CN,zh;q=0.9',
    'cache-control': 'max-age=0',
    'priority': 'u=0, i',
    'referer': 'https://qiye.qizhidao.com/lib/a01b00c00d00e00f00g00h00i00k00/sc2d10dc5af79548fbff85fbd1b22bf982/',
    'sec-ch-ua': '"Chromium";v="136", "Google Chrome";v="136", "Not.A/Brand";v="99"',
    'sec-ch-ua-mobile': '?0',
    'sec-ch-ua-platform': '"Windows"',
    'sec-fetch-dest': 'document',
    'sec-fetch-mode': 'navigate',
    'sec-fetch-site': 'same-origin',
    'sec-fetch-user': '?1',
    'upgrade-insecure-requests': '1',
    'user-agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/136.0.0.0 Safari/537.36',
    # 'cookie': 'sajssdk_2015_cross_new_user=1; sensorsdata2015jssdkchannel=%7B%22prop%22%3A%7B%22_sa_channel_landing_url%22%3A%22%22%7D%7D; wz_uuid=X/fe36031e2352d91d31adfe63416a305b; Hm_lvt_9ea3e7293b7c088e0d2c88874b63e7dd=1747617767; HMACCOUNT=A735DDA42AC02E6E; SSO_SESSION_ID=ZWVjOGY0MmYtZDExMi00YWZmLTg4ZWQtZDdlNDVkYjMwMWY0; ticket=eyJhbGciOiJIUzUxMiJ9.ZXlKNmFYQWlPaUpFUlVZaUxDSmhiR2NpT2lKa2FYSWlMQ0psYm1NaU9pSkJNVEk0UTBKRExVaFRNalUySW4wLi5odVhicExqVDBmTG1STXQ2b1ZPYnZnLkp2R3BYa2FlcHVaN1ZjUHkySjBheVRNVlZyWURSdmhmazREdWhrc2s2RTQwQWZvVDNhZXA0REkyRVF1ek1zekJOZEM2STFqZzJOSTJjRV9GX3VRZlZDNVZDWEFQSmtrS2tMSTZpSW5CM0FjYU82TFNjV2hrUTVva1htN29UTU9SbjZCVWpmcDdyRnl3RG5XRHo2NDRiSm1vT28xMGx2VnFGczM2OUlmeUltVmlfZjFmMU5GaHVYTk1mSzJrd2lVX2xGaUR5YThmd3dLZFpCaG1Wa3NMN3ktWnVMNDIzOWk3aUJYSHpTTkw5NjJSZVI1aTBPNjJyeEk1SUZmeWtrS0IuRGd4aE1TSE5YcHNRdlRiX3VZVXpxUQ.K0Ytme5n24LKre-EXu0nz7CClT1iT2HX_bt_vmFnppc3xwzHCGia43d3WyE3JZVksuo_N5EjcDxj3s3HgI-W8g; _g_t_=96f67a30d3211b400f55ae4ed110f261; token=eyJhbGciOiJIUzUxMiJ9.ZXlKNmFYQWlPaUpFUlVZaUxDSmhiR2NpT2lKa2FYSWlMQ0psYm1NaU9pSkJNVEk0UTBKRExVaFRNalUySW4wLi5odVhicExqVDBmTG1STXQ2b1ZPYnZnLkp2R3BYa2FlcHVaN1ZjUHkySjBheVRNVlZyWURSdmhmazREdWhrc2s2RTQwQWZvVDNhZXA0REkyRVF1ek1zekJOZEM2STFqZzJOSTJjRV9GX3VRZlZDNVZDWEFQSmtrS2tMSTZpSW5CM0FjYU82TFNjV2hrUTVva1htN29UTU9SbjZCVWpmcDdyRnl3RG5XRHo2NDRiSm1vT28xMGx2VnFGczM2OUlmeUltVmlfZjFmMU5GaHVYTk1mSzJrd2lVX2xGaUR5YThmd3dLZFpCaG1Wa3NMN3ktWnVMNDIzOWk3aUJYSHpTTkw5NjJSZVI1aTBPNjJyeEk1SUZmeWtrS0IuRGd4aE1TSE5YcHNRdlRiX3VZVXpxUQ.K0Ytme5n24LKre-EXu0nz7CClT1iT2HX_bt_vmFnppc3xwzHCGia43d3WyE3JZVksuo_N5EjcDxj3s3HgI-W8g; accessToken=eyJhbGciOiJIUzUxMiJ9.ZXlKNmFYQWlPaUpFUlVZaUxDSmhiR2NpT2lKa2FYSWlMQ0psYm1NaU9pSkJNVEk0UTBKRExVaFRNalUySW4wLi5odVhicExqVDBmTG1STXQ2b1ZPYnZnLkp2R3BYa2FlcHVaN1ZjUHkySjBheVRNVlZyWURSdmhmazREdWhrc2s2RTQwQWZvVDNhZXA0REkyRVF1ek1zekJOZEM2STFqZzJOSTJjRV9GX3VRZlZDNVZDWEFQSmtrS2tMSTZpSW5CM0FjYU82TFNjV2hrUTVva1htN29UTU9SbjZCVWpmcDdyRnl3RG5XRHo2NDRiSm1vT28xMGx2VnFGczM2OUlmeUltVmlfZjFmMU5GaHVYTk1mSzJrd2lVX2xGaUR5YThmd3dLZFpCaG1Wa3NMN3ktWnVMNDIzOWk3aUJYSHpTTkw5NjJSZVI1aTBPNjJyeEk1SUZmeWtrS0IuRGd4aE1TSE5YcHNRdlRiX3VZVXpxUQ.K0Ytme5n24LKre-EXu0nz7CClT1iT2HX_bt_vmFnppc3xwzHCGia43d3WyE3JZVksuo_N5EjcDxj3s3HgI-W8g; creditNo=%22%22; param_sign=roYuB3; Hm_lpvt_9ea3e7293b7c088e0d2c88874b63e7dd=1747619300; smidV2=202505191016307c10a18d5e8a15a50dda3fc1039d489f00ea7a21ddcd9e570; _bl_uid=smmtqayquXdgR4fq7006g2h9IX2I; Hm_lvt_8b5ea374777742fd73d0561bd1dfca6d=1747620994; acw_tc=f5d36357174762107277210433481bd19eb244908197885b08a7a18a2916d7; sensorsdata2015jssdkcross=%7B%22distinct_id%22%3A%22f7b1dcc0ddbf455d833fa99414f2780e%22%2C%22first_id%22%3A%22196e622d7c2e7c-0287f8b21c83c1e-26011f51-1327104-196e622d7c31b05%22%2C%22props%22%3A%7B%22%24latest_traffic_source_type%22%3A%22%E7%9B%B4%E6%8E%A5%E6%B5%81%E9%87%8F%22%2C%22%24latest_search_keyword%22%3A%22%E6%9C%AA%E5%8F%96%E5%88%B0%E5%80%BC_%E7%9B%B4%E6%8E%A5%E6%89%93%E5%BC%80%22%2C%22%24latest_referrer%22%3A%22%22%7D%2C%22identities%22%3A%22eyIkaWRlbnRpdHlfY29va2llX2lkIjoiMTk2ZTYyMmQ3YzJlN2MtMDI4N2Y4YjIxYzgzYzFlLTI2MDExZjUxLTEzMjcxMDQtMTk2ZTYyMmQ3YzMxYjA1IiwiJGlkZW50aXR5X2xvZ2luX2lkIjoiZjdiMWRjYzBkZGJmNDU1ZDgzM2ZhOTk0MTRmMjc4MGUifQ%3D%3D%22%2C%22history_login_id%22%3A%7B%22name%22%3A%22%24identity_login_id%22%2C%22value%22%3A%22f7b1dcc0ddbf455d833fa99414f2780e%22%7D%2C%22%24device_id%22%3A%22196e622d7c2e7c-0287f8b21c83c1e-26011f51-1327104-196e622d7c31b05%22%7D; x-web-ip=171.218.206.136, 47.121.110.187, 47.106.31.167, 100.121.99.209; .thumbcache_de0d870e3139ba2368b2e7ea8f11063c=hZVjd47Cv1sJHhGMmTvVn0f9DpivVn6IF/IytbVBOsjFSd0QEKKRar19TbnFCtp96Hi7/tlo0p6ovKPk4j5lOw%3D%3D; Hm_lpvt_8b5ea374777742fd73d0561bd1dfca6d=1747621872',
}

urls = []

with open("./urls.txt", "r", encoding="utf-8") as file:
    urls = [url.strip() for url in file.readlines()]

items = []
lock = threading.Lock()  # 用于线程安全的列表操作


def scrape_page(url, index):
    try:
        response = requests.get(
            url=url,
            cookies=cookies,
            headers=headers,
            timeout=10
        )
        response.raise_for_status()  # 检查请求是否成功

        tree = etree.HTML(response.text)
        list_1 = []

        # 提取数据的代码保持不变
        Companyname_number = tree.xpath(
            '//div[@class="copy-box initialShowCopyIcon"]/span/span[@class="text-num"]/text()')[0]
        Belonging_company = tree.xpath(
            '//div[@class="cell flex-center value"]/span/span[@class="test"]/text()')[6]
        if bool(re.search(r'\d', Belonging_company)):
            Belonging_company = tree.xpath(
                '//div[@class="cell flex-center value"]/span/span[@class="test"]/text()')[5]
        else:
            pass
        try:
            uscc = tree.xpath(
                '//div[@class="cell flex-center value fddb"]/span[@class="isPerson"]/a/text()')[0]
        except:
            uscc = tree.xpath(
                '//div[@class="cell flex-center value fddb"]/span/span[@class="test"]/text()')[0]

        Number_employees = tree.xpath(
            '//div[@class="cell flex-center value"]/span/span[@class="test"]/text()')[7]
        if bool(re.search(r'\d', Number_employees)):
            pass
        else:
            Number_employees = tree.xpath(
                '//div[@class="cell flex-center value"]/span/span[@class="test"]/text()')[6]
        operating_revenue = tree.xpath(
            '//div[@class="trade"]//div[@class="value"]/span/text()')[0]
        Establishment_time = tree.xpath(
            '//div[@class="cell flex-center value f-1"]/span/span/text()')[0]
        registered_capital = tree.xpath(
            '//div[@class="cell flex-center value"]/span/span[@class="test"]/text()')[1]
        Paid_capital = tree.xpath(
            '//div[@class="cell flex-center value"]/span/span[@class="test"]/text()')[2]
        Enterprise_type = tree.xpath(
            '//div[@class="cell flex-center value"]/span/span[@class="test"]/text()')[4]
        if '纳税人' in Enterprise_type:
            Enterprise_type = tree.xpath(
                '//div[@class="cell flex-center value"]/span/span[@class="test"]/text()')[3]
        else:
            pass
        try:
            registered_address = tree.xpath(
                '//div[@class="copy-box initialShowCopyIcon"]/span/div[@class="filter-source"]/text()')[0]
        except:
            registered_address = tree.xpath(
                '//div[@class="copy-box initialShowCopyIcon"]/span/div[@class="filter-source no-value"]/text()')[0]
        brief_introduction = tree.xpath(
            '//div[@class="company-detail-item"]/div[@class="text-box"]/div/p/text()')[0]
        business_scope = tree.xpath(
            '//div[@class="flex"]/p[@class="value f-1"]/text()')[0]
        # 正则表达式模式：匹配labelTitle为"联系电话"的labelValue
        pattern = r'labelTitle:\s*"联系电话"[^}]*labelValue:\s*"([^"]+)"'

        # 查找所有匹配项
        matches = re.findall(pattern, response.text)

        list_1.extend([Companyname_number, Belonging_company, uscc, Number_employees,
                      operating_revenue, Establishment_time, registered_capital,
                      Paid_capital, Enterprise_type, registered_address, brief_introduction, business_scope, matches])

        with lock:
            items.append(list_1)
            print(f'已完成 {index}/{len(urls)}: {url}')

        # print(business_scope)

        return True

    except Exception as e:
        print(f'处理 {url} 时出错: {str(e)}')
        return False

# 多线程爬取


def main():
    start_time = time.time()

    # 使用线程池，max_workers控制并发线程数
    with ThreadPoolExecutor(max_workers=10) as executor:
        # 提交所有任务
        futures = {executor.submit(scrape_page, url, i): (url, i)
                   for i, url in enumerate(urls, 1)}

        # 等待所有任务完成
        for future in as_completed(futures):
            url, i = futures[future]
            try:
                future.result()
            except Exception as e:
                print(f'任务异常: {url}, 错误: {e}')

    end_time = time.time()
    print(f"所有任务完成，耗时: {end_time - start_time:.2f}秒")
    print("$"*30)

    # 保存数据到Excel
    df = pd.DataFrame(items, columns=['统一社会信用代码', '所属行业', '法定代表人', '员工人数',
                                      '营业收入', '成立时间', '注册资本', '实缴资本',
                                      '企业类型', '注册地址', '简介', '经营范围', '联系电话'])
    df.to_excel('jichuxinxi改.xlsx', index=False)
    print("数据已保存到 jichuxinxi改.xlsx")
    print(items)


if __name__ == '__main__':
    main()
